import db from './db/inedx.js'
import { crawlWeb } from '../../utils/crawler.js'
import { saveJson, sleep, random } from '../../utils/util.js'
import { domain, isDev, dataDir, techniqueDir } from './config/index.js'
import { htmlToText } from 'html-to-text'

import Page from './page.js'

class Technique {
  constructor() {
    this.page = new Page()
  }

  get db() {
    return db.projectConfig
  }

  get config() {
    return db.projectConfig.data
  }

  async init() {}

  // 开始任务
  async start() {
    await this.crawList()
  }

  // 爬取列表
  async crawList() {
    const { $ } = await crawlWeb('https://attack.mitre.org/matrices/enterprise/')
    const lists = []

    $('.matrix .technique a').each(function () {
      $(this).find('sub').remove()
      const url = domain + $(this).attr('href')
      const title = $(this).text().trim()
      const id = url.split('/').pop()
      lists.push({
        id,
        url,
        title,
      })
    })

    saveJson(`${dataDir}/technique.json`, lists)

    const total = lists.length
    let count = 0

    for (let i = this.config.techniqueIndex; i < lists.length; i++) {
      const index = i + 1

      // 当前进度，加上百分比
      console.log(`当前进度:${index}/${total} ${(index / total) * 100}% ${lists[i].url} `)

      const content = lists[i]
      await this.crawlContent(content.url, content)

      this.config.techniqueIndex += 1
      await this.db.write()

      // 调试模式，只爬取前3个
      if (isDev) {
        count++
        if (count > 0) break
      }
    }
  }

  // 爬取详情
  async crawlContent(url, item) {
    await sleep(random(5000, 10000)) // 延迟1-5秒执行
    const { $ } = await request(
      url
      // { timeout: 1000 * 60 }
    )

    const json = this.page.getContentJson($)

    Object.assign(item, json)

    saveJson(`${techniqueDir}/${item.id}.json`, item)

    if (isDev) console.log('内容', item)

    return item
  }
}

export default Technique
